{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "['Current_1', 'Current_2', 'Current_3'，'Vibration_1']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import joblib\n",
    "import os\n",
    "from joblib import Parallel, delayed"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.sensor文件整合"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 常量\n",
    "new_input_dir = '../raw_data/test_add/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def integrate_sensors(data_no, csv_nos):\n",
    "    '''\n",
    "    描述：\n",
    "        合并所有的sensor文件，并分别输出\n",
    "    '''\n",
    "    if not os.path.exists('./sensors'):\n",
    "        os.mkdir('./sensors')\n",
    "        \n",
    "    new_input_subdir = new_input_dir + '0%d/Sensor/'%data_no\n",
    "    new_cols = ['Vibration_1','Vibration_2','Vibration_3', 'Current_1']\n",
    "    \n",
    "    def basis_func(idx):\n",
    "        new_sensor = pd.read_csv(new_input_subdir + '%d.csv'%idx)\n",
    "        new_sensor.columns = new_cols\n",
    "        sensor = new_sensor\n",
    "        \n",
    "        if not os.path.exists('./sensors/0%d'%data_no):\n",
    "            os.mkdir('./sensors/0%d'%data_no)\n",
    "        joblib.dump(sensor, './sensors/0%d/%d.lz4'%(data_no, idx), compress='lz4')\n",
    "                  \n",
    "    Parallel(n_jobs=10, verbose=1)(delayed(basis_func)(i) for i in range(1,csv_nos+1))\n",
    "   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers.\n",
      "[Parallel(n_jobs=10)]: Done   2 out of  10 | elapsed:   16.3s remaining:  1.1min\n",
      "[Parallel(n_jobs=10)]: Done  10 out of  10 | elapsed:   18.6s finished\n"
     ]
    }
   ],
   "source": [
    "integrate_sensors(1, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers.\n",
      "[Parallel(n_jobs=10)]: Done   2 out of  10 | elapsed:   19.6s remaining:  1.3min\n",
      "[Parallel(n_jobs=10)]: Done  10 out of  10 | elapsed:   22.3s finished\n"
     ]
    }
   ],
   "source": [
    "integrate_sensors(2, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers.\n",
      "[Parallel(n_jobs=10)]: Done   2 out of  10 | elapsed:   17.8s remaining:  1.2min\n",
      "[Parallel(n_jobs=10)]: Done  10 out of  10 | elapsed:   19.5s finished\n"
     ]
    }
   ],
   "source": [
    "integrate_sensors(3, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers.\n",
      "[Parallel(n_jobs=10)]: Done   2 out of  10 | elapsed:   22.3s remaining:  1.5min\n",
      "[Parallel(n_jobs=10)]: Done  10 out of  10 | elapsed:   23.6s finished\n"
     ]
    }
   ],
   "source": [
    "integrate_sensors(4, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers.\n",
      "[Parallel(n_jobs=10)]: Done   2 out of  10 | elapsed:   14.7s remaining:   58.9s\n",
      "[Parallel(n_jobs=10)]: Done  10 out of  10 | elapsed:   19.1s finished\n"
     ]
    }
   ],
   "source": [
    "integrate_sensors(5, 10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 以下的代码不要"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 新旧sensor不对齐"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def integrate_oddsensor(old_sensor, new_sensor):\n",
    "    \n",
    "#     old_cols = ['Current_1','Current_2','Current_3', 'Vibration_1']\n",
    "#     new_cols = ['Vibration_1','Vibration_2','Vibration_3', 'Current_1']\n",
    "\n",
    "#     old_sensor.columns = old_cols\n",
    "#     new_sensor.columns = new_cols\n",
    "#     if (old_sensor.index.shape[0] != new_sensor.index.shape[0]) or (old_sensor.index !=new_sensor.index).any():\n",
    "#         raise ValueError('index 不一致！')\n",
    "#     sensor = pd.concat([old_sensor[['Current_1', 'Current_2', 'Current_3']], \n",
    "#                         new_sensor[['Vibration_1', 'Vibration_2', 'Vibration_3']]], axis=1)\n",
    "    \n",
    "#     sensor['sort_col'] = sensor.index\n",
    "#     return sensor "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1. sensor_01_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# old_path = '../raw_data/test_poL3/01/Sensor/2.csv'\n",
    "# new_path = '../raw_data/test_add/01/Sensor/2.csv'\n",
    "# old_sensor = pd.read_csv(open(old_path)) \n",
    "# new_sensor = pd.read_csv(open(new_path))\n",
    "\n",
    "# set(new_sensor['current']).difference(set(old_sensor['vibration_1']))\n",
    "\n",
    "# new_sensor = new_sensor[(new_sensor['current'] != -36.849256636599996) & (new_sensor['current'] != -36.3260614122)]\n",
    "# new_sensor.reset_index(drop=True, inplace=True)\n",
    "\n",
    "# joblib.dump(integrate_oddsensor(old_sensor, new_sensor), './sensors/01/2.lz4', compress='lz4')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2.sensor_04_7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# old_path = '../raw_data/test_poL3/04/Sensor/7.csv'\n",
    "# new_path = '../raw_data/test_add/04/Sensor/7.csv'\n",
    "# old_sensor = pd.read_csv(open(old_path)) \n",
    "# new_sensor = pd.read_csv(open(new_path))\n",
    "\n",
    "# old_sensor = old_sensor.dropna(subset=['vibration_1']).reset_index(drop=True)\n",
    "# new_sensor = new_sensor.dropna(subset=['current']).reset_index(drop=True)\n",
    "\n",
    "# set(new_sensor['current']).difference(set(old_sensor['vibration_1']))\n",
    "\n",
    "# cond_1 = np.array([i in [ -23.614932459099997, -5.550522904099999, 1.2830953724, 41.545859199] for i in new_sensor['current']])\n",
    "# cond_2 = new_sensor['current'].isnull().values\n",
    "# cond = cond_1|cond_2\n",
    "# new_sensor = new_sensor[cond == False]\n",
    "# new_sensor.reset_index(drop=True, inplace=True)\n",
    "\n",
    "# new_vc_sorted = new_sensor['current'].value_counts().sort_index()\n",
    "# old_vc_sorted = old_sensor['vibration_1'].value_counts().sort_index()\n",
    "\n",
    "# dval =  new_vc_sorted - old_vc_sorted\n",
    "\n",
    "# # new_sensor: 692157, 693146, 696985\n",
    "# dval_idx = dval[dval!=0].index\n",
    "\n",
    "# new_sensor.drop(index=[692157, 693146, 696985], axis=0, inplace=True)\n",
    "# new_sensor.reset_index(drop=True, inplace=True)\n",
    "\n",
    "# joblib.dump(integrate_oddsensor(old_sensor, new_sensor), './sensors/04/7.lz4', compress='lz4')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
